'''
Created on 2013-5-14

@author: zengyu.cw
'''

# -*- coding:utf-8 -*-

from sgmllib import SGMLParser
from urlparse import urlparse
import re
import urllib
import sys
import os
sys.path.append(os.getcwd())
import LogFactory
    
itemPtn = re.compile(r"\d+")
logger = LogFactory.getLogger("loggerName")

class URLLister(SGMLParser):
    
    def __init__(self):
        SGMLParser.__init__(self)
            
    def reset(self):
        self.urls = []
        SGMLParser.reset(self)
        
    def start_a(self, attrs):
        hrefs = [v for k,v in attrs if k == 'href' and getmainurl(v) == "http://ju.taobao.com"]
        if hrefs:
            self.urls.extend(hrefs) 
            
def bfs(source, depth):
    count = 1
    q = source
    targetURLs = []
    while count <= depth:
        logger.error(str(count) + " " + str(len(q)))
        #print q
        tmp = addURLs(q)
        targetURLs.extend(tmp)
        count += 1
        q = tmp
    #return targetURLs
    return [item for item in targetURLs if getItemId(item)]
    
def addURLs(sources):
    #print "add sources"
    #print sources
    result = []
    urllister = URLLister()
    count = 0
    for item in sources:
        logger.info(str(count) + " open url " + item)
        count += 1
        if getItemId(item):
            result.extend(item)
            continue
        try:
            socket = urllib.urlopen(item)
            try:
                urllister.feed(socket.read())
            finally:
                socket.close()
        except:
            continue
        #finally:
            #socket.close()
    result.extend(list(set(urllister.urls)))
    return result

def getmainurl(url):
    ind = url.find('/',len('http://'))
    if ind > 0 :
        return url[:ind]
    else :
        return url
    
def getItemId(query):
    values = urlparse(query).query.split("&")
    for item in values:
        kvs = item.split("=")
        if kvs[0] in ["id","itemId","item_id"] and len(kvs) == 2 and re.match(itemPtn,kvs[1]):
            return kvs[1]
    return None
        
        

if __name__ == '__main__':
    result = bfs(["http://ju.taobao.com"],5)
    print "result" + str(len(result))
    logger.error([msg for msg in result])